{
    class Rl {
        constructor (o) {
            this._o = o;
            this.qtable = new Qtable({
                actions: o.actions, // available actions
                states: o.states // state length
            });
            this.lr = o.learningRate;
            this.gamma = o.rewardDecay;
            this.epsilon = o.eGreedy;
        }

        chooseAction(observation) {
            this._checkStateExist(observation);

            let actionName = null;
            let maxEle = this.qtable.max({
                state: observation,
                action: "__require__",
            });

            // action selection
            if ( Math.random() < this.epsilon ) {
                // choose best action
                actionName = maxEle.name;
            } else { 
                // choose random action
                actionName = this.qtable.random("actions");
            }
            return actionName;
        }

        learn(state, action, reward, _state, _action) {
            this._checkStateExist(_state);
            let qPredict = this.qtable.q(state, action).value;
            let qTarget = 0;
            if ( _state != "__terminal__" ) {
                qTarget = reward + this.gamma * this.qtable.q(_state, _action).value;//this.qtable.max({state: _state,action: "__require__"}).value;
            } else {
                qTarget = reward;
            }
            let oldVal = this.qtable.q(state, action).value;
            this.qtable.q(state, action).set( oldVal + this.lr * (qTarget - qPredict) );
        }

        _checkStateExist(state) {
            if ( !this.qtable.in("state", state) ) {
                this.qtable.append("state", state);
            }
        }

        getQtabel() {
            return this.qtable.getTable();
        }

    }

    window.Rl = Rl;
}